
import matplotlib.pyplot as plt
import os
from utils import  get_accuracy, get_samples, train_out_proj_fast, train_out_proj_closed_form
from main import Args
from data import load_SHD
from model import EchoSpike, simple_out
import numpy as np
from data import augment_shd
import torch
import seaborn as sns
from scipy.signal import savgol_filter
from tqdm.notebook import trange
from matplotlib import pyplot
pyplot.rcParams['figure.dpi'] = 600
import pickle
torch.manual_seed(0)
color_list = sns.color_palette('muted')
device = 'cpu'
batch_size = 64
folder = 'models/'
model_name = folder + 'shd_4layer_withckpts_noaugment.pt'
with open(model_name[:-3] + '_args.pkl', 'rb') as f:
    args = pickle.load(f)
# args = Args()
online = args.online
print(vars(args))

/home/lars/miniconda3/lib/python3.9/site-packages/torchvision/datapoints/__init__.py:12: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().
  warnings.warn(_BETA_TRANSFORMS_WARNING)
/home/lars/miniconda3/lib/python3.9/site-packages/torchvision/transforms/v2/__init__.py:54: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().
  warnings.warn(_BETA_TRANSFORMS_WARNING)

{'model_name': 'shd_4layer_withckpts_noaugment', 'dataset': 'shd', 'online': True, 'device': 'cuda', 'recurrency_type': 'none', 'lr': 0.0001, 'epochs': 1000, 'augment': False, 'batch_size': 128, 'n_hidden': [450, 450, 450, 450], 'inp_thr': 0.05, 'c_y': [1.5, -1.5], 'n_inputs': 700, 'n_outputs': 20, 'n_time_bins': 100, 'beta': 0.95}


#train_loader, test_loader = load_PMNIST(n_time_bins, scale=0.9, patches=True) #load_NMNIST(n_time_bins, batch_size=batch_size)
n_time_bins = 100
train_loader, test_loader = load_SHD(batch_size=batch_size) #load_NMNIST(n_time_bins, batch_size=batch_size)
# Plot Example(s)
for i in range(1):
    frames, target = train_loader.next_item(-1, contrastive=True)
    plt.figure(figsize=(10, 10))
    plt.axis('off')
    plt.imshow(frames.squeeze(1).T)
    # plt.colorbar()
    print(frames.shape, target)
plt.axis('on')

/home/lars/ownCloud/ETH/Master/Project_2/SNN_CLAPP/data.py:28: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  self.y = torch.tensor(y)

torch.Size([100, 1, 700]) tensor([4.])

(-0.5, 99.5, 699.5, -0.5)


SNN = EchoSpike(args.n_inputs, args.n_hidden, beta=args.beta, c_y=args.c_y, device=device, recurrency_type=args.recurrency_type, online=args.online).to(device)
SNN.load_state_dict(torch.load(model_name, map_location=device))
# train(SNN, train_loader, args.epochs, device, args.model_name,
                            # batch_size=args.batch_size, online=args.online, lr=1e-8, augment=args.augment)
from_epoch = 0
echo_train_loss = torch.load(model_name[:-3]+'_loss_hist.pt', map_location='cpu')[int(from_epoch*len(train_loader)/args.batch_size):]
print(echo_train_loss.shape)
for i in range(echo_train_loss.shape[-1]):
    plt.plot(from_epoch+(args.batch_size*np.arange(echo_train_loss.shape[0])/len(train_loader)), savgol_filter(echo_train_loss[:,i], 99, 1), color=color_list[i])
plt.legend([f'layer {i+1}' for i in range(len(SNN.layers))])
# no y ticks, because it's not really meaningful
plt.yticks([])
plt.xlabel('Epoch')
plt.ylabel('EchoSpike Loss')

torch.Size([63719, 4])

Text(0, 0.5, 'EchoSpike Loss')


# plotting adaptive threshold and update rate for an example
# init_echo, label_0 = train_loader.next_item(-1, contrastive=True)
# sample_1, label_1 = train_loader.next_item(-1, contrastive=True)
# sample_2, label_2 = train_loader.next_item(label_1, contrastive=False)
print(label_0, label_1, label_2)
SNN.eval()
with torch.no_grad():
    # feed first sample to get initial activity
    for t in range(100):
        inp_activity = init_echo[t].mean(axis=-1)
        SNN(init_echo[t], torch.tensor(-1, device=device), inp_activity=inp_activity)
    SNN.reset(-1)
    # feed second sample to get the update rates and thresholds for contrastive case
    contrastive_thresholds = torch.zeros(100)
    contrastive_temp_sim = torch.zeros((len(SNN.layers), 100))
    for t in range(100):
        inp_activity = sample_1[t].mean(axis=-1)
        out_spk, mems, losses = SNN(sample_1[t], torch.tensor(-1, device=device), inp_activity=inp_activity)
        contrastive_thresholds[t] = inp_activity * args.c_y[1]
        contrastive_temp_sim[:, t] = losses
    SNN.reset(-1)
    # feed third sample to get the update rates and thresholds for predictive case
    predictive_thresholds = torch.zeros(100)
    predictive_temp_sim = torch.zeros((len(SNN.layers), 100))
    for t in range(100):
        inp_activity = sample_2[t].mean(axis=-1)
        out_spk, mems, losses = SNN(sample_1[t], torch.tensor(1, device=device), inp_activity=inp_activity)
        predictive_thresholds[t] = inp_activity * args.c_y[0]
        predictive_temp_sim[:, t] = -losses
    SNN.reset(1)
    # plot thresholds, with sample as background
    layer = 2
    fig, ax = plt.subplots(figsize=(10, 5))
    ax2  = ax.twinx()
    # imshow in background
    ax.imshow(sample_1.squeeze(1).T, aspect='auto', cmap='Reds')
    ax2.plot(-contrastive_temp_sim[layer], color='r', label='Negative Similarity Score')
    ax2.plot(contrastive_thresholds, color='r', linestyle='--', label='Contrastive Threshold')
    ax2.hlines(args.inp_thr*args.c_y[1], 0, 100, color='r', linestyle=':', label='Input Threshold (times c(-1))')
    # highlight regions where the thresholds are crossed
    argwhere = np.argwhere(np.logical_and((-contrastive_temp_sim[layer] < contrastive_thresholds).numpy(), contrastive_thresholds.numpy() < args.inp_thr*args.c_y[1]))
    for i in range(argwhere.shape[0]):
        ax2.axvspan(argwhere[i], argwhere[i]+1, color='r', alpha=0.2, lw=0)

    ax.yaxis.set_visible(False)
    ax2.spines['right'].set_visible(False)
    ax2.yaxis.tick_left()
    ax2.yaxis.set_label_position('left')
    ax2.set_xlim(ax.get_xlim())
    # get rid of right margin
    ax2.margins(0)
    ax.set_xlabel('Timesteps')
    plt.ylabel('Thresholds & Similarity Score')
    plt.xlim(0, 100)
    plt.legend()
    # same for predictive
    fig, ax = plt.subplots(figsize=(10, 5))
    ax2  = ax.twinx()
    # imshow in background
    ax.imshow(sample_2.squeeze(1).T, aspect='auto', cmap='Blues')
    ax2.plot(predictive_temp_sim[layer], color='b', label='Similarity Score')
    ax2.plot(predictive_thresholds, color='b', linestyle='--', label='Predictive Threshold')
    ax2.hlines(args.inp_thr*args.c_y[0], 0, 100, color='b', linestyle=':', label='Input Threshold (times c(1))')
    # highlight regions where the thresholds are crossed
    argwhere = np.argwhere(np.logical_and((predictive_temp_sim[layer] < predictive_thresholds).numpy(), predictive_thresholds.numpy() > args.inp_thr*args.c_y[0]))
    for i in range(argwhere.shape[0]):
        ax2.axvspan(argwhere[i], argwhere[i]+1, color='b', alpha=0.1, lw=0)
    ax.yaxis.set_visible(False)
    ax2.spines['right'].set_visible(False)
    ax2.yaxis.tick_left()
    ax2.yaxis.set_label_position('left')
    ax2.set_xlim(ax.get_xlim())
    # get rid of right margin
    #ax2.margins(0)
    ax.set_xlabel('Timesteps')
    plt.ylabel('Thresholds & Similarity Score')
    plt.xlim(0, 100)
    plt.legend()
    plt.show()

tensor([9.]) tensor([7.]) tensor([7.])

/home/lars/miniconda3/lib/python3.9/site-packages/matplotlib/patches.py:1111: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
  xy = np.asarray(xy)
/home/lars/miniconda3/lib/python3.9/site-packages/matplotlib/patches.py:1111: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
  xy = np.asarray(xy)


layers = [SNN.layers[0].fc.weight[:,:args.n_inputs]]
for i in range(1, len(SNN.layers)):
    layers.append(SNN.layers[i].fc.weight[:,:args.n_hidden[i-1]] @ layers[-1])

for i in range(len(SNN.layers)):
    plt.figure()
    plt.imshow(SNN.layers[i].fc.weight.detach(), vmax=0.1, vmin=-0.1)
    plt.colorbar()
    # plt.figure()
    # plt.imshow(SNN.layers[i].pred.weight.detach(), vmax=0.5, vmin=-0.5)
    # plt.colorbar()
for lay in layers:
    plt.figure()
    plt.imshow(lay.detach())
    plt.colorbar()

Error in callback <function flush_figures at 0x7f47940f8700> (for post_execute):

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
File ~/miniconda3/lib/python3.9/site-packages/PIL/ImageFile.py:495, in _save(im, fp, tile, bufsize)
    494 try:
--> 495     fh = fp.fileno()
    496     fp.flush()

AttributeError: '_idat' object has no attribute 'fileno'

During handling of the above exception, another exception occurred:

KeyboardInterrupt                         Traceback (most recent call last)
File ~/miniconda3/lib/python3.9/site-packages/matplotlib_inline/backend_inline.py:126, in flush_figures()
    123 if InlineBackend.instance().close_figures:
    124     # ignore the tracking, just draw and close all figures
    125     try:
--> 126         return show(True)
    127     except Exception as e:
    128         # safely show traceback if in IPython, else raise
    129         ip = get_ipython()

File ~/miniconda3/lib/python3.9/site-packages/matplotlib_inline/backend_inline.py:90, in show(close, block)
     88 try:
     89     for figure_manager in Gcf.get_all_fig_managers():
---> 90         display(
     91             figure_manager.canvas.figure,
     92             metadata=_fetch_figure_metadata(figure_manager.canvas.figure)
     93         )
     94 finally:
     95     show._to_draw = []

File ~/miniconda3/lib/python3.9/site-packages/IPython/core/display_functions.py:298, in display(include, exclude, metadata, transient, display_id, raw, clear, *objs, **kwargs)
    296     publish_display_data(data=obj, metadata=metadata, **kwargs)
    297 else:
--> 298     format_dict, md_dict = format(obj, include=include, exclude=exclude)
    299     if not format_dict:
    300         # nothing to display (e.g. _ipython_display_ took over)
    301         continue

File ~/miniconda3/lib/python3.9/site-packages/IPython/core/formatters.py:178, in DisplayFormatter.format(self, obj, include, exclude)
    176 md = None
    177 try:
--> 178     data = formatter(obj)
    179 except:
    180     # FIXME: log the exception
    181     raise

File <decorator-gen-2>:2, in __call__(self, obj)

File ~/miniconda3/lib/python3.9/site-packages/IPython/core/formatters.py:222, in catch_format_error(method, self, *args, **kwargs)
    220 """show traceback on failed format call"""
    221 try:
--> 222     r = method(self, *args, **kwargs)
    223 except NotImplementedError:
    224     # don't warn on NotImplementedErrors
    225     return self._check_return(None, args[0])

File ~/miniconda3/lib/python3.9/site-packages/IPython/core/formatters.py:339, in BaseFormatter.__call__(self, obj)
    337     pass
    338 else:
--> 339     return printer(obj)
    340 # Finally look for special method names
    341 method = get_real_method(obj, self.print_method)

File ~/miniconda3/lib/python3.9/site-packages/IPython/core/pylabtools.py:151, in print_figure(fig, fmt, bbox_inches, base64, **kwargs)
    148     from matplotlib.backend_bases import FigureCanvasBase
    149     FigureCanvasBase(fig)
--> 151 fig.canvas.print_figure(bytes_io, **kw)
    152 data = bytes_io.getvalue()
    153 if fmt == 'svg':

File ~/miniconda3/lib/python3.9/site-packages/matplotlib/backend_bases.py:2319, in FigureCanvasBase.print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2315 try:
   2316     # _get_renderer may change the figure dpi (as vector formats
   2317     # force the figure dpi to 72), so we need to set it again here.
   2318     with cbook._setattr_cm(self.figure, dpi=dpi):
-> 2319         result = print_method(
   2320             filename,
   2321             facecolor=facecolor,
   2322             edgecolor=edgecolor,
   2323             orientation=orientation,
   2324             bbox_inches_restore=_bbox_inches_restore,
   2325             **kwargs)
   2326 finally:
   2327     if bbox_inches and restore_bbox:

File ~/miniconda3/lib/python3.9/site-packages/matplotlib/backend_bases.py:1648, in _check_savefig_extra_args.<locals>.wrapper(*args, **kwargs)
   1640     _api.warn_deprecated(
   1641         '3.3', name=name, removal='3.6',
   1642         message='%(name)s() got unexpected keyword argument "'
   1643                 + arg + '" which is no longer supported as of '
   1644                 '%(since)s and will become an error '
   1645                 '%(removal)s')
   1646     kwargs.pop(arg)
-> 1648 return func(*args, **kwargs)

File ~/miniconda3/lib/python3.9/site-packages/matplotlib/_api/deprecation.py:412, in delete_parameter.<locals>.wrapper(*inner_args, **inner_kwargs)
    402     deprecation_addendum = (
    403         f"If any parameter follows {name!r}, they should be passed as "
    404         f"keyword, not positionally.")
    405     warn_deprecated(
    406         since,
    407         name=repr(name),
   (...)
    410                  else deprecation_addendum,
    411         **kwargs)
--> 412 return func(*inner_args, **inner_kwargs)

File ~/miniconda3/lib/python3.9/site-packages/matplotlib/backends/backend_agg.py:541, in FigureCanvasAgg.print_png(self, filename_or_obj, metadata, pil_kwargs, *args)
    494 """
    495 Write the figure to a PNG file.
    496 
   (...)
    538     *metadata*, including the default 'Software' key.
    539 """
    540 FigureCanvasAgg.draw(self)
--> 541 mpl.image.imsave(
    542     filename_or_obj, self.buffer_rgba(), format="png", origin="upper",
    543     dpi=self.figure.dpi, metadata=metadata, pil_kwargs=pil_kwargs)

File ~/miniconda3/lib/python3.9/site-packages/matplotlib/image.py:1675, in imsave(fname, arr, vmin, vmax, cmap, format, origin, dpi, metadata, pil_kwargs)
   1673 pil_kwargs.setdefault("format", format)
   1674 pil_kwargs.setdefault("dpi", (dpi, dpi))
-> 1675 image.save(fname, **pil_kwargs)

File ~/miniconda3/lib/python3.9/site-packages/PIL/Image.py:2212, in Image.save(self, fp, format, **params)
   2209         fp = builtins.open(filename, "w+b")
   2211 try:
-> 2212     save_handler(self, fp, filename)
   2213 finally:
   2214     # do what we can to clean up
   2215     if open_fp:

File ~/miniconda3/lib/python3.9/site-packages/PIL/PngImagePlugin.py:1348, in _save(im, fp, filename, chunk, save_all)
   1346     _write_multiple_frames(im, fp, chunk, rawmode)
   1347 else:
-> 1348     ImageFile._save(im, _idat(fp, chunk), [("zip", (0, 0) + im.size, 0, rawmode)])
   1350 if info:
   1351     for info_chunk in info.chunks:

File ~/miniconda3/lib/python3.9/site-packages/PIL/ImageFile.py:509, in _save(im, fp, tile, bufsize)
    507 else:
    508     while True:
--> 509         l, s, d = e.encode(bufsize)
    510         fp.write(d)
    511         if s:

KeyboardInterrupt:


from tqdm.notebook import tqdm
def train_out_proj(epochs, batch, cat, out_projs=None):
    # train output projections from all layers (and no layer)
    losses_out = []
    beta = 1.0
    lr = 1e-4
    augment = True
    optimizers = []
    print_interval = 10*batch
    if out_projs is None:
        out_projs = []
        out_proj_0 = simple_out(700, 20, beta=beta)
    else:
        for out_p in out_projs:
            out_p.train()
            out_p.reset()
        out_proj_0 = out_projs[0]
        out_projs = out_projs[1:]
    optim_0 = torch.optim.Adam(out_proj_0.parameters(), lr=lr)
    for lay in range(len(SNN.layers)):
        if len(out_projs) <= lay:
            if cat:
                out_projs.append(simple_out(sum(args.n_hidden[:lay+1])+700, 20, beta=beta))
            else:
                out_projs.append(simple_out(args.n_hidden[lay], 20, beta=beta))
        optimizers.append(torch.optim.Adam(out_projs[lay].parameters(), lr=lr))
        optimizers[-1].zero_grad()
    SNN.eval()
    acc = []
    target = batch_size*[-1]
    correct = (len(SNN.layers) + 1)*[0]
    with torch.no_grad():
        pbar = tqdm(total=len(train_loader)*epochs)
        while len(losses_out)*batch < len(train_loader)*epochs:
            data, target = train_loader.next_item(target, contrastive=True)
            SNN.reset(0)
            logit_lists = [[] for _ in range(len(SNN.layers)+1)]
            data = data.squeeze()
            if augment:
                data = augment_shd(data)
            for step in range(data.shape[0]):
                data_step = data[step].float().to(device)
                target = target.to(device)
                logits, _, _ = SNN(data_step, 0)
                if step == args.n_time_bins-1:
                    _, logts = out_proj_0(data_step, target)
                    logit_lists[0] = logts
                    for lay in range(len(SNN.layers)):
                        if cat:
                            data_step = torch.cat([data_step, logits[lay]], dim=-1)
                        else:
                            data_step = logits[lay]
                        _, logts = out_projs[lay](data_step, target)
                        logit_lists[lay+1] = logts
                else:
                    out_proj_0(data_step, None)
                    for lay in range(len(SNN.layers)):
                        if cat:
                            data_step = torch.cat([data_step, logits[lay]], dim=-1)
                        else:
                            data_step = logits[lay]
                        out_projs[lay](data_step, None)
            
            preds = [logit_lists[lay].argmax(axis=-1) for lay in range(len(SNN.layers)+1)]
            correct = [correct[lay] + (preds[lay] == target).sum() for lay in range(len(SNN.layers)+1)]
            out_proj_0.reset()
            for i, out_proj in enumerate(out_projs):
                out_proj.reset()

            losses_out.append(torch.tensor([torch.nn.functional.cross_entropy(logit_lists[lay], target.squeeze().long()) for lay in range(len(SNN.layers)+1)], requires_grad=False))

            optim_0.step()
            optim_0.zero_grad()
            for opt in optimizers:
                opt.step()
                opt.zero_grad()
            
            if len(losses_out)*batch % print_interval == 0:
                pbar.write(f'Cross Entropy Loss: {(torch.stack(losses_out)[-print_interval//batch:].sum(dim=0)/(print_interval//batch)).numpy()}\n' +
                           f'Correct: {100*np.array(correct)/print_interval}%')
                acc.append(np.array(correct)/print_interval)
                correct = (len(SNN.layers) + 1)*[0]
            pbar.update(batch)
    return [out_proj_0, *out_projs], np.asarray(acc), torch.stack(losses_out)

with torch.no_grad():
    if args.augment:
        n_epochs = 100
        cat = True
        # if already trained, load the output projections
        if os.path.exists(model_name[:-3]+'_out_projs.pt'):
            out_projs = torch.load(model_name[:-3]+'_out_projs.pt', map_location=device)
        else:
            out_projs, acc, losses_out = train_out_proj(n_epochs, batch_size, cat)
            torch.save(out_projs, model_name[:-3]+'_out_projs.pt')


if not args.augment:
    # if snn_samples already exist, don't recompute
    try:
        snn_samples
    except:
        snn_samples, targets = get_samples(SNN, train_loader, args.n_hidden, device)
    cat = True
    test_accs = []
    train_accs = []
    for i in range(10):
        with torch.no_grad():
            out_projs, acc, losses_out = train_out_proj_fast(SNN, args, 30, 60, snn_samples, train_loader.y, cat=cat, lr=3e-4, weight_decay=0)
        print('Mean abs weights', out_projs[-1].out_proj.weight.abs().mean())
        test_accs.append(get_accuracy(SNN, out_projs, test_loader, device, cat=cat)[0])
        train_accs.append(get_accuracy(SNN, out_projs, train_loader, device, cat=cat)[0])
    test_accs = torch.stack([torch.tensor(ta) for ta in test_accs])
    train_accs = torch.stack([torch.tensor(ta) for ta in train_accs])
    print(f'Fast Classifier Mean Test Accuracy: {100*torch.mean(test_accs, dim=0)}, Std: {100*torch.std(test_accs, dim=0)}')
    print(f'Fast Classifier Mean Train Accuracy: {100*torch.mean(train_accs, dim=0)}, Std: {100*torch.std(train_accs, dim=0)}')

  0%|          | 0/30 [00:00<?, ?it/s]

Cross Entropy Loss: [4.1872716 3.1436896 2.324911  1.8332611 1.7116711]
Correct: [15.91466405 28.84992643 41.88327612 53.29818538 56.47376165]%
Cross Entropy Loss: [2.463053   1.4402385  0.9932619  0.78709584 0.7274044 ]
Correct: [29.61010299 52.52574792 66.63805787 74.35017165 76.85139774]%
Cross Entropy Loss: [2.1605024  1.1310219  0.75093096 0.6065797  0.5781179 ]
Correct: [35.49534085 61.73369299 74.46051986 79.78175576 80.78715056]%
Cross Entropy Loss: [1.9743724  0.98842406 0.6655614  0.52868205 0.4756272 ]
Correct: [40.10544385 66.84649338 78.05296714 82.74889652 83.88916135]%
Cross Entropy Loss: [1.8470854  0.86089724 0.5742121  0.44715923 0.39279133]
Correct: [43.79597842 71.21137813 81.37567435 85.63021089 87.02795488]%
Cross Entropy Loss: [1.7041218  0.7355151  0.4549692  0.36407518 0.30456907]
Correct: [45.63511525 75.14713095 85.26238352 87.86169691 89.66405101]%
Cross Entropy Loss: [1.6372793  0.69917977 0.42999554 0.3589159  0.29322165]
Correct: [47.27807749 76.77783227 85.66699362 88.03334968 90.17900932]%
Cross Entropy Loss: [1.544904  0.6397305 0.4042687 0.2856387 0.2902844]
Correct: [50.73565473 78.64149093 86.75821481 90.57135851 90.39970574]%
Cross Entropy Loss: [1.4992337  0.561572   0.34116337 0.2432679  0.20581922]
Correct: [50.94409024 80.93428151 89.02648357 92.21432075 93.12162825]%
Cross Entropy Loss: [1.4029716  0.5246412  0.30427465 0.24020304 0.22663431]
Correct: [53.82540461 82.24619912 90.31387935 92.09171162 92.71701815]%
Cross Entropy Loss: [1.3696545  0.47617233 0.27150342 0.21061409 0.1887534 ]
Correct: [54.48749387 84.67385974 91.47866601 93.37910741 93.84502207]%
Cross Entropy Loss: [1.3367599  0.46279103 0.23594631 0.1949752  0.17442143]
Correct: [56.04462972 84.64933791 92.55762629 94.01667484 94.1760667 ]%
Cross Entropy Loss: [1.2394575  0.42377707 0.2390069  0.19376123 0.1666555 ]
Correct: [58.01863659 85.96125552 92.5698872  93.63658656 94.4825895 ]%
Cross Entropy Loss: [1.2089213  0.3754208  0.19392681 0.15874341 0.12948033]
Correct: [59.10985777 87.45708681 94.1760667  94.7768514  96.03972536]%
Cross Entropy Loss: [1.1781707  0.37796757 0.19277297 0.13802704 0.1132929 ]
Correct: [61.16969103 87.27317312 94.04119667 95.72094164 96.39529181]%
Cross Entropy Loss: [1.1654673  0.35604432 0.19146349 0.14440024 0.13116387]
Correct: [60.5443845  88.37665522 94.00441393 95.40215792 96.11329083]%
Cross Entropy Loss: [1.0976348  0.30079675 0.15331014 0.11085778 0.08470406]
Correct: [62.21186856 90.58361942 95.72094164 96.51790093 97.47425208]%
Cross Entropy Loss: [1.0685236  0.29694378 0.14392856 0.1122706  0.06621685]
Correct: [64.21039725 90.77979402 95.73320255 96.56694458 98.24668955]%
Cross Entropy Loss: [1.0360285  0.28520513 0.12562898 0.09520219 0.07301001]
Correct: [64.51692006 91.09857773 96.65277097 97.10642472 97.76851398]%
Cross Entropy Loss: [0.9997829  0.27274328 0.11821497 0.08664852 0.07347295]
Correct: [65.35066209 91.34379598 96.8857283  97.21677293 97.75625307]%
Cross Entropy Loss: [0.98923266 0.25430578 0.11565954 0.08620941 0.07150187]
Correct: [65.44874939 92.34919078 96.93477195 97.40068661 97.85434036]%
Cross Entropy Loss: [0.9962766  0.23677072 0.12023983 0.06941527 0.06794292]
Correct: [66.38057872 92.50858264 96.71407553 98.25895047 98.12408043]%
Cross Entropy Loss: [0.93273515 0.23093246 0.09307903 0.06428653 0.06228227]
Correct: [68.15841099 93.18293281 97.74399215 98.34477685 98.32025503]%
Cross Entropy Loss: [0.9235989  0.23393106 0.10302919 0.0937719  0.0582747 ]
Correct: [68.25649828 92.39823443 97.36390387 96.95929377 98.33251594]%
Cross Entropy Loss: [0.9203787  0.2202786  0.09355038 0.05742612 0.03718374]
Correct: [68.86954389 92.88867092 97.58460029 98.55321236 99.0926925 ]%
Cross Entropy Loss: [0.9017033  0.18595962 0.06985462 0.05069679 0.05567232]
Correct: [69.13928396 94.5806768  98.52869053 98.7984306  98.35703776]%
Cross Entropy Loss: [0.8860314  0.20338023 0.08454282 0.07205418 0.06439789]
Correct: [69.47032859 93.74693477 97.65816577 97.73173124 97.82981854]%
Cross Entropy Loss: [0.85007566 0.16172642 0.06845957 0.05027504 0.05842838]
Correct: [70.89259441 95.65963708 98.44286415 98.71260422 98.27121138]%
Cross Entropy Loss: [0.84972966 0.15339603 0.06147075 0.06692296 0.06831121]
Correct: [71.21137813 96.03972536 98.68808239 97.90338401 98.1976459 ]%
Cross Entropy Loss: [0.86331654 0.19340706 0.08035859 0.03712239 0.04313882]
Correct: [71.10102992 94.94850417 98.09955861 99.21530162 98.83521334]%
Mean abs weights tensor(0.0239, grad_fn=<MeanBackward0>)

  0%|          | 0/36 [00:00<?, ?it/s]

Directly from inputs:
Accuracy: 45.80%
From layer 1:
Accuracy: 67.89%
From layer 2:
Accuracy: 74.78%
From layer 3:
Accuracy: 75.18%
From layer 4:
Accuracy: 77.08%

  0%|          | 0/128 [00:00<?, ?it/s]

Directly from inputs:
Accuracy: 71.38%
From layer 1:
Accuracy: 95.02%
From layer 2:
Accuracy: 99.15%
From layer 3:
Accuracy: 99.68%
From layer 4:
Accuracy: 96.17%

  0%|          | 0/30 [00:00<?, ?it/s]

Cross Entropy Loss: [4.2056136 3.3595018 2.5578706 1.8632474 1.7972437]
Correct: [16.00049044 27.78322707 40.5590976  53.23688082 55.21088769]%
Cross Entropy Loss: [2.5309207 1.482339  1.038259  0.8312534 0.7516103]
Correct: [29.27905836 51.98626778 65.21579205 73.01373222 76.17704757]%
Cross Entropy Loss: [2.2038283  1.1757452  0.7548449  0.63051116 0.5463139 ]
Correct: [34.64933791 60.97351643 74.35017165 79.03384012 81.65767533]%
Cross Entropy Loss: [1.9805763  0.9832883  0.6415817  0.55061483 0.4850844 ]
Correct: [39.2594409  66.98136341 78.39627268 82.13585091 84.37959784]%
Cross Entropy Loss: [1.8289003  0.85267454 0.55737317 0.43624127 0.38633227]
Correct: [43.08484551 71.44433546 81.57184895 85.20107896 87.45708681]%
Cross Entropy Loss: [1.7603719  0.7941842  0.47148356 0.38691586 0.34094897]
Correct: [45.16920059 73.46738597 84.12211869 87.07699853 88.47474252]%
Cross Entropy Loss: [1.6479888  0.6881737  0.4219232  0.33900353 0.29466173]
Correct: [47.6949485  76.76557136 85.89995096 89.02648357 90.53457577]%
Cross Entropy Loss: [1.5874764  0.6404749  0.3832031  0.3667489  0.31605873]
Correct: [49.301128   78.22461991 87.22412948 87.95978421 89.94605199]%
Cross Entropy Loss: [1.5148996  0.5874625  0.31833452 0.2530897  0.22447881]
Correct: [50.94409024 80.05149583 89.36978911 91.76066699 92.82736636]%
Cross Entropy Loss: [1.3929242  0.517211   0.2957919  0.23422898 0.19924119]
Correct: [53.911231   82.78567925 90.24031388 92.32466896 93.56302109]%
Cross Entropy Loss: [1.3655514  0.46343637 0.26418233 0.18656996 0.17288554]
Correct: [55.33349681 84.29377146 91.90779794 94.18832761 94.60519863]%
Cross Entropy Loss: [1.3460088  0.44891194 0.2609801  0.18687023 0.16876407]
Correct: [55.4193232  85.01716528 91.20892594 94.01667484 94.71554684]%
Cross Entropy Loss: [1.2780318  0.42506018 0.23402876 0.16550633 0.1458253 ]
Correct: [56.96419814 86.02256008 92.72927906 94.96076508 95.61059343]%
Cross Entropy Loss: [1.1999463  0.37642714 0.2073232  0.17451656 0.16576539]
Correct: [59.22020598 87.71456596 93.64884747 94.47032859 94.76459049]%
Cross Entropy Loss: [1.1591609  0.3495398  0.18536784 0.13290927 0.11321939]
Correct: [60.25012261 88.94065718 94.10250123 95.92937715 96.46885728]%
Cross Entropy Loss: [1.1474069  0.345629   0.19094934 0.1126137  0.09575427]
Correct: [61.26777832 88.80578715 94.06571849 96.75085826 97.08190289]%
Cross Entropy Loss: [1.1157053  0.32503557 0.16931117 0.14722033 0.10864975]
Correct: [62.59195684 89.54144188 95.03433055 95.85581167 96.70181462]%
Cross Entropy Loss: [1.1233001  0.3216199  0.17507222 0.12341614 0.12036664]
Correct: [62.27317312 89.32074546 94.74006866 96.11329083 96.06424718]%
Cross Entropy Loss: [1.084442   0.27813444 0.15503414 0.1219961  0.07550655]
Correct: [63.24178519 91.24570868 95.26728789 96.10102992 97.75625307]%
Cross Entropy Loss: [1.0297537  0.27438426 0.11474238 0.07875086 0.06496222]
Correct: [64.82344286 91.58901422 96.81216282 97.96468857 98.13634134]%
Cross Entropy Loss: [1.0179255  0.26768327 0.12178159 0.10539071 0.09340606]
Correct: [65.43648847 91.31927415 96.53016184 96.84894556 96.89798921]%
Cross Entropy Loss: [0.9677667  0.23042907 0.09970601 0.08376812 0.08283429]
Correct: [67.05492889 93.06032369 97.68268759 97.53555665 97.52329573]%
Cross Entropy Loss: [0.93068594 0.22338991 0.08591006 0.06659058 0.03558221]
Correct: [67.83962727 93.42815105 98.14860226 98.30799411 99.38695439]%
Cross Entropy Loss: [0.92695904 0.21457809 0.08566278 0.1193421  0.04286044]
Correct: [68.64884747 93.56302109 97.85434036 96.59146641 98.85973516]%
Cross Entropy Loss: [0.9117389  0.20471463 0.07515663 0.06993123 0.03466035]
Correct: [68.62432565 94.1760667  98.45512506 98.02599313 99.11721432]%
Cross Entropy Loss: [0.91249263 0.23524472 0.1092992  0.09192672 0.07934052]
Correct: [67.81510544 92.58214811 97.06964198 97.26581658 97.53555665]%
Cross Entropy Loss: [0.9042726  0.1978319  0.09377304 0.07481949 0.11384055]
Correct: [69.10250123 94.10250123 97.43746935 97.70720942 97.02059833]%
Cross Entropy Loss: [0.8498304  0.18166998 0.08278241 0.03884922 0.07832021]
Correct: [70.45120157 94.43354586 97.90338401 99.17851888 97.65816577]%
Cross Entropy Loss: [0.8266773  0.16396347 0.08591412 0.03990128 0.10072411]
Correct: [71.6895537  95.50024522 97.57233938 99.16625797 96.87346739]%
Cross Entropy Loss: [0.8071401  0.15629496 0.06097132 0.06256734 0.06781789]
Correct: [72.24129475 95.6841589  98.63903874 98.17312408 97.75625307]%
Mean abs weights tensor(0.0242, grad_fn=<MeanBackward0>)

  0%|          | 0/36 [00:00<?, ?it/s]

Directly from inputs:
Accuracy: 45.05%
From layer 1:
Accuracy: 69.08%
From layer 2:
Accuracy: 71.91%
From layer 3:
Accuracy: 73.19%
From layer 4:
Accuracy: 74.87%


# run all checkpoints
if not args.augment: 
    last_ckp = 20
    test_accs_ckpt = []
    train_accs_ckpt = []
    epochs = []
    while True:
        print(f'Checkpoint {last_ckp}')
        SNN_ckp = EchoSpike(args.n_inputs, args.n_hidden, beta=args.beta, device=device, recurrency_type=args.recurrency_type, online=args.online).to(device)
        try:
            print(model_name[:-3] + f'_epoch{last_ckp}.pt')
            state_dict = torch.load(model_name[:-3] + f'_epoch{last_ckp}.pt', map_location=device)
            # state_dict = {key.replace('clapp', 'layers'):value for key, value in state_dict.items()}
            # torch.save(state_dict, model_name + f'_epoch{last_ckp}.pt')
            SNN_ckp.load_state_dict(state_dict)
        except:
            if last_ckp > 1500:
                break
            else:
                last_ckp += 20
                continue
        epochs.append(last_ckp)
        last_ckp += 20
        snn_samples, targets = get_samples(SNN_ckp, train_loader, args.n_hidden, device)
        cat = True
        with torch.no_grad():
            out_projs, acc, losses_out = train_out_proj_fast(SNN_ckp, args, 60, 60, snn_samples, train_loader.y, cat=cat, lr=1e-4, weight_decay=1)
        test_accs_ckpt.append(torch.tensor(get_accuracy(SNN_ckp, out_projs, test_loader, device, cat=cat)[0]))
        train_accs_ckpt.append(torch.tensor(get_accuracy(SNN_ckp, out_projs, train_loader, device, cat=cat)[0])) 
    test_accs_ckpt = torch.stack(test_accs_ckpt)
    train_accs_ckpt = torch.stack(train_accs_ckpt)
    # save the results
    torch.save(torch.stack([torch.tensor(tac) for tac in test_accs_ckpt]), model_name[:-3]+'_test_accs_ckpt.pt')
    torch.save(torch.stack([torch.tensor(tac) for tac in train_accs_ckpt]), model_name[:-3]+'_train_accs_ckpt.pt')


# plot train and test accuracy over time
plt.figure()
for i in range(test_accs_ckpt.shape[-1]):
    if i == 0:
        plt.plot(epochs, 100*test_accs_ckpt[:,i], color=color_list[i], label=f'Directly from inputs')
        plt.plot(epochs, 100*train_accs_ckpt[:,i], color=color_list[i], linestyle='--')
    else:
        plt.plot(epochs, 100*test_accs_ckpt[:,i], color=color_list[i], label=f'Layer {i}')
        plt.plot(epochs, 100*train_accs_ckpt[:,i], color=color_list[i], linestyle='--')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

<matplotlib.legend.Legend at 0x7f4d60de9850>


if not args.augment:
    # if snn_samples already exist, don't recompute
    try:
        snn_samples
    except:
        snn_samples, targets = get_samples(SNN, train_loader, args.n_hidden, device)
    cat = True
    out_projs_closed = train_out_proj_closed_form(args, snn_samples, targets, cat=cat)
    test_acc_closed, _ = get_accuracy(SNN, out_projs_closed, test_loader, device, cat=cat)
    train_acc_closed, _ = get_accuracy(SNN, out_projs_closed, train_loader, device, cat=cat)

    # grouped Bar plot the Accuracies of the different layers both during training and testing
    sns.set_theme(style="whitegrid")
    labels = ['From Inputs Directly', *[f'Until Layer {i+1}' for i in range(len(SNN.layers))]]
    if not cat:
        labels = ['From Inputs Directly', *[f'From Layer {i+1}' for i in range(len(SNN.layers))]]
    x = np.arange(len(labels))  # the label locations
    width = 0.35  # the width of the bars
    fig, ax = plt.subplots()
    rects1 = ax.bar(x - width/2, 100*torch.tensor(test_acc_closed), width, label='Test Accuracy', color=color_list[0])
    rects2 = ax.bar(x + width/2, 100*torch.tensor(train_acc_closed), width, label='Train Accuracy', color=color_list[1])
    # remove horizontal lines and spines
    ax.spines['right'].set_visible(False)
    ax.spines['left'].set_visible(False)
    ax.xaxis.grid(False)
    plt.xticks(np.arange(len(out_projs_closed)), labels, rotation=45)
    plt.legend()
    plt.ylabel('Accuracy [%]')
    plt.ylim([25, 100])

(20, 700) 0.010724677 -0.010629931
(20, 1150) 0.056771163 -0.039086737
(20, 1600) 0.4956497 -0.2596547
(20, 2050) 0.44425982 -0.25678265
(20, 2500) 0.8378215 -0.91546047

  0%|          | 0/36 [00:00<?, ?it/s]

Directly from inputs:
Accuracy: 28.89%
From layer 1:
Accuracy: 55.79%
From layer 2:
Accuracy: 67.67%
From layer 3:
Accuracy: 74.20%
From layer 4:
Accuracy: 75.88%

  0%|          | 0/128 [00:00<?, ?it/s]

Directly from inputs:
Accuracy: 66.96%
From layer 1:
Accuracy: 93.44%
From layer 2:
Accuracy: 97.95%
From layer 3:
Accuracy: 99.13%
From layer 4:
Accuracy: 99.50%


if args.augment:
    # plot some training characteristics
    print(f'Accuracy of last quarter: {100*acc[-len(acc)//4:].mean(axis=0)}%')
    plt.figure()
    for i in range(len(acc[0])):
        plt.plot(np.asarray(acc)[:,i]*100, color=color_list[i])
    plt.ylabel('Accuracy [%]')
    plt.xlabel('Training Step [x500]')
    labels = ['From Inputs directly', *[f'From Layer {i+1}' for i in range(len(SNN.layers))]]
    plt.legend(labels)
    plt.ylim([65, 95])
    plt.figure()
    print(losses_out.shape)
    for i in range(losses_out.shape[1]):
        plt.plot(np.arange(len(losses_out))/len(train_loader), savgol_filter(losses_out[:,i], 99, 1), label=labels[i], color=color_list[i])
    plt.ylabel('Cross Entropy Loss')
    plt.xlabel('Training Step')
    plt.ylim([0.15, 1.0])
    plt.legend();

Accuracy of last quarter: [43.03571429 77.07000589 87.01358909 89.62446036 90.12460754]%
torch.Size([25488, 5])


test_acc, pred_matrix = get_accuracy(SNN, out_projs, test_loader, device, cat=cat)
plt.figure()
plt.plot(100*np.asarray(test_acc))
plt.ylabel('Accuracy [%]')
plt.xlabel('Layer')

plt.figure()
plt.imshow(pred_matrix, origin='lower')
plt.title('Prediction Matrix for the final layer')
plt.xlabel('Prediction')
plt.ylabel('Target')
plt.xticks([i for i in range(args.n_outputs)])
plt.yticks([i for i in range(args.n_outputs)])
plt.colorbar();

  0%|          | 0/36 [00:00<?, ?it/s]

Directly from inputs:
Accuracy: 46.64%
From layer 1:
Accuracy: 73.14%
From layer 2:
Accuracy: 78.80%
From layer 3:
Accuracy: 80.17%
From layer 4:
Accuracy: 80.96%


from utils import get_accuracy
if args.augment:
    train_acc, _ = get_accuracy(SNN, out_projs, train_loader, device, cat=cat) 
else:
    test_acc = torch.mean(test_accs, dim=0)
    print(test_acc)
    train_acc = torch.mean(train_accs, dim=0)
# grouped Bar plot the Accuracies of the different layers both during training and testing
sns.set_theme(style="whitegrid")
labels = ['From Inputs Directly', *[f'Until Layer {i+1}' for i in range(len(SNN.layers))]]
if not cat:
    labels = ['From Inputs Directly', *[f'From Layer {i+1}' for i in range(len(SNN.layers))]]
x = np.arange(len(labels))  # the label locations
width = 0.35  # the width of the bars
fig, ax = plt.subplots()
rects1 = ax.bar(x - width/2, 100*torch.tensor(test_acc), width, label='Test Accuracy', color=color_list[0])
rects2 = ax.bar(x + width/2, 100*torch.tensor(train_acc), width, label='Train Accuracy', color=color_list[1])
if not args.augment:
    ax.errorbar(x - width/2, 100*test_acc, yerr=100*torch.std(test_accs, dim=0), fmt='none', capsize=6, color=color_list[3])
    ax.errorbar(x + width/2, 100*train_acc, yerr=100*torch.std(train_accs, dim=0), fmt='none', capsize=6, color=color_list[3])
# remove horizontal lines and spines
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(False)
ax.xaxis.grid(False)
plt.xticks(np.arange(len(out_projs)), labels, rotation=45)
plt.legend()
plt.ylabel('Accuracy [%]')
plt.ylim([25, 100])
#plt.title('SHD Accuracy');

tensor([0.4502, 0.6792, 0.7394, 0.7537, 0.7663])

/tmp/ipykernel_901041/1496231988.py:16: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  rects1 = ax.bar(x - width/2, 100*torch.tensor(test_acc), width, label='Test Accuracy', color=color_list[0])
/tmp/ipykernel_901041/1496231988.py:17: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
  rects2 = ax.bar(x + width/2, 100*torch.tensor(train_acc), width, label='Train Accuracy', color=color_list[1])

(25.0, 100.0)


# Randomly select k sample of each class and save the spiking activity
n_outputs = 20
n_repeats = 1
k = 20
fewshot_accuracies = torch.zeros((n_repeats, len(SNN.layers)))
for n in range(n_repeats):
    SNN.reset(0)
    one_shot_samples = torch.zeros(n_outputs, n_time_bins, n_inputs)
    one_shot_spks = torch.zeros(n_outputs, len(SNN.layers), n_hidden[0])
    for i in trange(n_outputs):
        for j in range(k):
            img, _ = train_loader.next_item(i, contrastive=False)
            one_shot_samples[i] = img.squeeze()
            for t in range(n_time_bins):
                logits, mem_his, clapp_loss = SNN(img[t].float(), 0) 
                one_shot_spks[i] += torch.stack(logits).squeeze()

    def metric(spk, one_shot):
        dists = torch.zeros(spk.shape[0], args.n_outputs)
        for i in range(args.n_outputs):
            one_shot_i = one_shot[i] / one_shot[i].sum()
            dists[:, i] = torch.einsum('bi, i->b' , spk, one_shot_i)
        return dists

    def get_predictions(spks):
        preds = torch.zeros(len(spks), spks[0].shape[0])
        # for each layer get the prediction
        for i in range(len(spks)):
            dists = metric(spks[i], one_shot_spks[:,i])
            preds[i] = dists.argmax(axis=-1)
        return preds
    dataset = test_loader
    batch = int(len(dataset)/100)
    correct_oneshot = torch.zeros(len(SNN.layers))
    SNN.eval()
    pred_matrix_oneshot = torch.zeros(n_outputs, n_outputs)
    for idx in trange(0, len(dataset), batch):
        SNN.reset(0)
        inp, target = dataset.x[idx:idx+batch], dataset.y[idx:idx+batch]
        logits = torch.zeros(len(SNN.layers), inp.shape[0], n_hidden[0])
        for step in range(inp.shape[1]):
            data_step = inp[:,step].float().to(device)
            spk_step, _, _ = SNN(data_step, 0)
            logits += torch.stack(spk_step)
        preds = get_predictions(logits)
        for i in range(preds.shape[0]):
            correct_oneshot[i] += int((preds[i] == target).sum())
        # for the last layer create the prediction matrix
        for j in range(preds.shape[1]):
            pred_matrix_oneshot[int(target[j]), int(preds[-1, j])] += 1
    correct_oneshot /= len(dataset)
    for i in range(len(SNN.layers)):
        print(f'From layer {i+1}:')
        print(f'Accuracy: {100*correct_oneshot[i]:.2f}%')
        fewshot_accuracies[n, i] = correct_oneshot[i]
    plt.imshow(pred_matrix_oneshot, origin='lower')
    plt.title('Prediction Matrix for the final layer')
    plt.xlabel('Prediction')
    plt.ylabel('Target')
    plt.xticks([i for i in range(n_outputs)])
    plt.yticks([i for i in range(n_outputs)])
    plt.colorbar();
    plt.figure()


# Boxplot of the accuracies
plt.figure()
sns.set_style("whitegrid")
g = sns.boxplot(data=fewshot_accuracies*100)
# remove left spines
sns.despine(left=True)
plt.xticks(np.arange(len(SNN.layers)), [f'Layer {i+1}' for i in range(len(SNN.layers))])
plt.ylabel('Few-Shot Test Accuracy [%]')
plt.ylim([0, 100])
print(f'Average Accuracy: {100*fewshot_accuracies.mean(axis=0)}%')
print(f'Maximum Accuracy: {fewshot_accuracies.max(axis=0)}%')

EchoSpike Predictive Plasticity¶

Dataset¶

Load pretrained model¶

Analyze Weights Directly¶

Train output Projection¶

Get output projection Accuracy on test set¶

Few Shot Learning (discontinued)¶